﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.IO.Compression;

namespace WebCrawler
{
    /// <summary>
    ///HtmlTool 的摘要说明
    /// </summary>
    public class HtmlTool
    {
        /// <summary>
        /// 获取网页的HTML内容，指定Encoding
        /// </summary>
        /// <param name="strUrl"></param>
        /// <param name="encoding"></param>
        /// <returns></returns>
        public static string GetHtml(string strUrl)
        {
            string strWebUrl = string.Empty;
            return GetHtml(strUrl, ref strWebUrl);
        }

        /// <summary>
        /// 获取百度页面
        /// </summary>
        /// <param name="strBaiDuUrl"></param>
        /// <returns></returns>
        public static string GetBaiDuHtml(string strBaiDuUrl)
        {
            string strHTML = string.Empty;
            try
            {
                HttpWebRequest requestPage = (HttpWebRequest)WebRequest.Create(new Uri(strBaiDuUrl));
                requestPage.Timeout = 10000;
                requestPage.ReadWriteTimeout = 20000;
                requestPage.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
                requestPage.UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:22.0) Gecko/20100101 Firefox/22.0";
                requestPage.Method = "GET";
                requestPage.Referer = "http://www.baidu.com";
                requestPage.Host = "www.baidu.com";
                //requestPage.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
                //requestPage.Headers.Add("Accept-Encoding", "gzip,deflate");
                HttpWebResponse responsePage = (HttpWebResponse)requestPage.GetResponse();
                Stream streamReceive = responsePage.GetResponseStream();
                using (StreamReader readerPage = new StreamReader(streamReceive, Encoding.GetEncoding("utf-8")))
                {
                    strHTML = readerPage.ReadToEnd();
                    readerPage.Dispose();
                }
                streamReceive.Dispose();
                responsePage.Close();
                if (string.IsNullOrEmpty(strHTML))
                {
                    Common.Log("baidu","获取页面失败:" + strBaiDuUrl + " 返回空");
                }
                return strHTML;
            }
            catch (Exception ex)
            {
                Common.Log("baidu","获取百度页面失败:" + strBaiDuUrl + " " + ex.Message.ToString());
                return "";
            }
        }

        /// <summary>
        /// 获取网页的HTML内容，指定Encoding
        /// </summary>
        /// <param name="strUrl"></param>
        /// <param name="encoding"></param>
        /// <returns></returns>
        public static string GetHtml(string strUrl, ref string strWebUrl)
        {
            string strHTML = string.Empty;
            try
            {
                string strDomain = GetWebUrl(strUrl);
                HttpWebRequest requestPage = (HttpWebRequest)WebRequest.Create(new Uri(strUrl));
                requestPage.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
                requestPage.UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:22.0) Gecko/20100101 Firefox/22.0";
                requestPage.Method = "GET";
                requestPage.Referer = "http://" + strDomain;
                requestPage.Host = strDomain;
                requestPage.Timeout = 10000;
                requestPage.ReadWriteTimeout = 20000;
                CookieContainer cc = new CookieContainer();
                requestPage.CookieContainer = cc;
                //requestPage.AllowWriteStreamBuffering = true;
                //requestPage.Credentials = System.Net.CredentialCache.DefaultCredentials;
                //requestPage.MaximumResponseHeadersLength = -1;
                requestPage.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
                requestPage.Headers.Add("Accept-Encoding", "gzip,deflate");
                HttpWebResponse responsePage = (HttpWebResponse)requestPage.GetResponse();
                strWebUrl = responsePage.ResponseUri.AbsoluteUri.Trim();
                Stream receiveStream = responsePage.GetResponseStream();
                if (responsePage.ContentEncoding.ToLower().Contains("gzip"))
                {
                    receiveStream = new GZipStream(receiveStream, CompressionMode.Decompress);
                }

                //获取网页字符编码描述信息 
                byte[] buffer = StreamToBytes(receiveStream);
                Encoding encode = Encoding.UTF8;
                strHTML = encode.GetString(buffer);
                string strEncod = string.Empty;
                Match charSetMatch = Regex.Match(strHTML, @"(?i)\bcharset=(?<charset>[-a-zA-Z_0-9]+)", RegexOptions.IgnoreCase | RegexOptions.Multiline);
                if (charSetMatch.Success)
                {
                    string webCharSet = charSetMatch.Groups["charset"].Value.Trim();
                    strEncod = webCharSet.ToLower();
                }
                else
                {

                    charSetMatch = Regex.Match(strHTML, "<meta([^<]*)charset=[\"'](?<charset>[-a-zA-Z_0-9]+)[\"']", RegexOptions.IgnoreCase | RegexOptions.Multiline);
                    if (charSetMatch.Success)
                    {
                        string webCharSet = charSetMatch.Groups["charset"].Value.Trim();
                        strEncod = webCharSet.ToLower();
                    }
                }
                if (string.IsNullOrEmpty(strEncod))
                {
                    strEncod=responsePage.CharacterSet.Trim().ToLower();
                }
                if (string.IsNullOrEmpty(strEncod))
                {
                    strEncod = "utf-8";
                }
                if (strEncod != "utf-8" && strEncod != "utf8")
                {
                    encode = Encoding.GetEncoding(strEncod);
                    strHTML = encode.GetString(buffer);
                }
                receiveStream.Close();
                buffer = null;

                if (string.IsNullOrEmpty(strHTML))
                {
                    Common.Log("获取页面失败:" + strUrl + " 返回空");
                }
                return strHTML.Trim();
            }
            catch (Exception ex)
            {
                Common.Log("获取页面失败:" + strUrl + " " + ex.Message.ToString());
                return "";
            }
        }

        /// <summary>
        ///  检查是否含有中文
        /// </summary>
        /// <param name="str">需要检查的字符串</param>
        /// <returns></returns>
        public static bool CheckCN(string str)
        {
            byte[] strASCII = System.Text.ASCIIEncoding.ASCII.GetBytes(str);
            int tmpNum = 0;
            for (int i = 0; i < str.Length; i++)
            {
                //中文检查
                if ((int)strASCII[i] >= 63 && (int)strASCII[i] < 91)
                {
                    tmpNum += 2;
                    break;
                }
            }
            if (tmpNum >= 2)
            {
                return true;
            }
            else
            {
                return false;
            }
        }

        /// <summary>
        /// 获取网站名称
        /// </summary>
        /// <param name="strUrl"></param>
        /// <returns></returns>
        public static string GetWebUrl(string strUrl)
        {
            if (strUrl.IndexOf("http://") == 0)
                strUrl = strUrl.Substring(7, strUrl.Length - 7);
            string[] strItem = strUrl.Split('/');
            return strItem[0].ToString();
        }

        /// <summary>
        /// 根据网页的HTML内容提取网页的Encoding
        /// </summary>
        /// <param name="strHtml"></param>
        /// <returns></returns>
        public static Encoding GetEncoding(string strHtml)
        {
            string pattern = @"(?i)\bcharset=(?<charset>[-a-zA-Z_0-9]+)";
            string charset = Regex.Match(strHtml, pattern).Groups["charset"].Value;
            try { return Encoding.GetEncoding(charset); }
            catch (ArgumentException) { return null; }
        }

        /// <summary>
        /// 根据网页的HTML内容提取网页的Title
        /// </summary>
        /// <param name="strUrl"></param>
        /// <returns></returns>
        public static string GetTitle(string strUrl)
        {
            string html = GetHtml(strUrl);
            string pattern = @"(?si)<title(?:\s+(?:""[^""]*""|'[^']*'|[^""'>])*)?>(?<title>.*?)</title>";
            return Regex.Match(html, pattern, RegexOptions.IgnoreCase | RegexOptions.Multiline).Groups["title"].Value.Trim();
        }

        /// <summary>
        /// 根据网页的HTML内容提取网页的Title
        /// </summary>
        /// <param name="strUrl"></param>
        /// <param name="strHtml"></param>
        /// <returns></returns>
        public static string GetTitleAndHtml(string strUrl, ref string strHtml)
        {
            strHtml = GetHtml(strUrl);
            string pattern = @"(?si)<title(?:\s+(?:""[^""]*""|'[^']*'|[^""'>])*)?>(?<title>.*?)</title>";
            return Regex.Match(strHtml, pattern, RegexOptions.IgnoreCase | RegexOptions.Multiline).Groups["title"].Value.Trim();
        }

        /// <summary>
        /// 根据网页的HTML内容提取网页的Title
        /// </summary>
        /// <param name="strUrl"></param>
        /// <param name="strHtml"></param>
        /// <returns></returns>
        public static string GetTitleAndBody(string strUrl, ref string strHtml)
        {
            strHtml = GetHtml(strUrl);
            string pattern = @"(?si)<title(?:\s+(?:""[^""]*""|'[^']*'|[^""'>])*)?>(?<title>.*?)</title>";
            string strTitle = Regex.Match(strHtml, pattern, RegexOptions.IgnoreCase | RegexOptions.Multiline).Groups["title"].Value.Trim();
            strHtml = NoHTML(strHtml);
            return strTitle;
        }

        /**/
        ///   <summary>
        ///   去除HTML标记
        ///   </summary>
        ///   <param   name="NoHTML">包括HTML的源码   </param>
        ///   <returns>已经去除后的文字</returns>
        public static string NoHTML(string strHTML)
        {
            //删除脚本
            strHTML = Regex.Replace(strHTML, @"<script[^>]*?>.*?</script>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            //删除HTML
            strHTML = Regex.Replace(strHTML, @"<(.[^>]*)>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = Regex.Replace(strHTML, @"([\r\n])[\s]+", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = Regex.Replace(strHTML, @"-->", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = Regex.Replace(strHTML, @"<!--.*", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = Regex.Replace(strHTML, @"&(quot|#34);", "\"", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = Regex.Replace(strHTML, @"&(amp|#38);", "&", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = Regex.Replace(strHTML, @"&(lt|#60);", "<", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = Regex.Replace(strHTML, @"&(gt|#62);", ">", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = Regex.Replace(strHTML, @"&(nbsp|#160);", "   ", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = Regex.Replace(strHTML, @"&(iexcl|#161);", "\xa1", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = Regex.Replace(strHTML, @"&(cent|#162);", "\xa2", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = Regex.Replace(strHTML, @"&(pound|#163);", "\xa3", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = Regex.Replace(strHTML, @"&ldquo;", "“", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = Regex.Replace(strHTML, @"&rdquo;", "”", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = Regex.Replace(strHTML, @"&(copy|#169);", "\xa9", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = Regex.Replace(strHTML, @"&#(\d+);", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            strHTML = strHTML.Replace("\r\n", "");
            return strHTML;
        }

        /// <summary> 
        /// 将 Stream 转成 byte[] 
        /// </summary> 
        private static byte[] StreamToBytes(Stream stream)
        {
            MemoryStream stmMemory = new MemoryStream();
            byte[] buffer = new byte[4096];
            int i;
            while ((i = stream.Read(buffer, 0, buffer.Length)) > 0)
            {
                stmMemory.Write(buffer, 0, i);
            }
            byte[] arraryByte = stmMemory.ToArray();
            stmMemory.Close();


            // 设置当前流的位置为流的开始 
            if (stream.CanSeek)
            {
                stream.Seek(0, SeekOrigin.Begin);
            }
            return arraryByte;
        }
    }
}